package org.example.sink;

import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.serialization.SimpleStringEncoder;
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.connector.source.util.ratelimit.RateLimiterStrategy;
import org.apache.flink.configuration.MemorySize;
import org.apache.flink.connector.datagen.source.DataGeneratorSource;
import org.apache.flink.connector.datagen.source.GeneratorFunction;
import org.apache.flink.connector.file.sink.FileSink;
import org.apache.flink.core.fs.Path;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.filesystem.OutputFileConfig;
import org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.DateTimeBucketAssigner;
import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.DefaultRollingPolicy;

import java.time.Duration;
import java.util.stream.Stream;

public class SinkFileDemo {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(2);
        //必需开启checkpoint;
        env.enableCheckpointing(2000, CheckpointingMode.EXACTLY_ONCE);

        DataGeneratorSource dataGeneratorSource = new DataGeneratorSource<>((GeneratorFunction<Long, String>) aLong -> "NUMBER:" + aLong,
                Long.MAX_VALUE, RateLimiterStrategy.perSecond(100), Types.STRING);

        DataStreamSource datagen = env.fromSource(dataGeneratorSource, WatermarkStrategy.noWatermarks(), "datagen");

        FileSink fileSink = FileSink
                //输出文件格式
                .forRowFormat(new Path("E:/tmp"), new SimpleStringEncoder<>("UTF-8"))
                //文件前缀、后缀
                .withOutputFileConfig(
                        OutputFileConfig.builder()
                                .withPartPrefix("json")
                                .withPartSuffix(".log")
                                .build()
                )
                //文件分桶策略
                .withBucketAssigner(new DateTimeBucketAssigner<>())
                .withRollingPolicy(DefaultRollingPolicy.builder()
                        .withInactivityInterval(Duration.ofSeconds(60))
                        .withMaxPartSize(new MemorySize(1024))
                        .build()
                )
                .build();

        datagen.sinkTo(fileSink);
        env.execute("SinkFileDemo");
    }
}
